{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用submitter对api进行访问\n",
    "from sparksampling import Submitter\n",
    "from sparksampling.var import FILE_TYPE_CSV\n",
    "from sparksampling.var import SIMPLE_RANDOM_SAMPLING_METHOD\n",
    "submitter = Submitter()\n",
    "dataset_uri = 'hdfs://localhost:9000/dataset/ten_million_top1k.csv'\n",
    "fraction = 0.1\n",
    "selected_features_list = ['X_20','X_80']\n",
    "label_index = 'y'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 17:59:51,754 - INFO - request: http://localhost:8000/v1/sampling/simplejob/ with data {'path': 'hdfs://localhost:9000/dataset/ten_million_top1k.csv', 'method': 'random', 'type': 'csv', 'with_header': True, 'conf': {'fraction': 0.1}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'code': 0, 'msg': '', 'data': {'job_id': 10024}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提交抽样任务\n",
    "submit_response = submitter.submit_sampling_simplejob(dataset_uri,\n",
    "                                              method=SIMPLE_RANDOM_SAMPLING_METHOD,\n",
    "                                              file_type=FILE_TYPE_CSV,\n",
    "                                              fraction=fraction,\n",
    "                                              with_header=True)\n",
    "job_id = submit_response.job_id\n",
    "submit_response.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:01:14,804 - INFO - request: http://localhost:8000/v1/query/sampling/job/ with data {'job_id': 10024}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'code': 0, 'msg': '', 'data': {'job_id': 10024, 'job_status': 'Succeed', 'msg': 'succeed', 'method': 'Simple Random Sampling', 'start_time': '2021/05/29/ 17:59:52', 'end_time': '2021/05/29 17:59:59', 'simpled_file_path': 'hdfs://localhost:9000/dataset/ten_million_top1k.csv-sampled-1622282391.7654526', 'request_data': \"{'path': 'hdfs://localhost:9000/dataset/ten_million_top1k.csv', 'method': 'random', 'type': 'csv', 'with_header': True, 'conf': {'fraction': 0.1, 'path': 'hdfs://localhost:9000/dataset/ten_million_top1k.csv', 'method': 'random', 'file_type': 'csv', 'with_header': True, 'seed': 58591, 'with_replacement': True, 'col_key': None}}\"}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'hdfs://localhost:9000/dataset/ten_million_top1k.csv-sampled-1622282391.7654526'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查询抽样任务\n",
    "sampling_job_details = submitter.get_sampling_job_details(job_id)\n",
    "sampled_path = sampling_job_details.sampled_path\n",
    "print(sampling_job_details.to_dict())\n",
    "sampled_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.85      0.89       517\n",
      "           1       0.86      0.92      0.89       483\n",
      "\n",
      "    accuracy                           0.89      1000\n",
      "   macro avg       0.89      0.89      0.89      1000\n",
      "weighted avg       0.89      0.89      0.89      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 简单的读取抽样之后的文件\n",
    "from pyspark.sql import SparkSession\n",
    "from sparksampling.config import SPARK_CONF\n",
    "\n",
    "conf = SPARK_CONF\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n",
    "df = spark.read.csv(sampled_path, header=True).toPandas()\n",
    "# 可以在这后面做数据分析，或试试看下面的统计、评估功能\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "train_y = df[label_index]\n",
    "train_X = df[selected_features_list]\n",
    "# train_X = df.drop([\"# id\"], axis=1)\n",
    "model.fit(train_X,train_y)\n",
    "tsdf = spark.read.csv(dataset_uri, header=True)\n",
    "tdf = tsdf.toPandas()\n",
    "\n",
    "test_y = tdf[label_index]\n",
    "test_X = tdf[train_X.columns]\n",
    "# test_X = test_X[feature_list]\n",
    "pred_y = model.predict(test_X)\n",
    "# data analyse here\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_true=test_y, y_pred=pred_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:02:17,964 - INFO - request: http://localhost:8000/v1/evaluation/statistics/ with data {'path': 'hdfs://localhost:9000/dataset/ten_million_top1k.csv', 'type': 'csv', 'method': 'basic', 'with_header': True, 'from_sampling': False}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th># id</th>\n",
       "      <th>X_0</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_2</th>\n",
       "      <th>X_3</th>\n",
       "      <th>X_4</th>\n",
       "      <th>X_5</th>\n",
       "      <th>X_6</th>\n",
       "      <th>X_7</th>\n",
       "      <th>...</th>\n",
       "      <th>X_91</th>\n",
       "      <th>X_92</th>\n",
       "      <th>X_93</th>\n",
       "      <th>X_94</th>\n",
       "      <th>X_95</th>\n",
       "      <th>X_96</th>\n",
       "      <th>X_97</th>\n",
       "      <th>X_98</th>\n",
       "      <th>X_99</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>count</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>...</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean</td>\n",
       "      <td>499.5</td>\n",
       "      <td>2.049</td>\n",
       "      <td>3.051</td>\n",
       "      <td>2.815</td>\n",
       "      <td>3.118</td>\n",
       "      <td>4.211</td>\n",
       "      <td>3.992</td>\n",
       "      <td>3.351</td>\n",
       "      <td>2.658</td>\n",
       "      <td>...</td>\n",
       "      <td>0.029485678112799993</td>\n",
       "      <td>-0.015578821739792</td>\n",
       "      <td>-0.034382649559500045</td>\n",
       "      <td>0.03410696849619999</td>\n",
       "      <td>0.004251966935300004</td>\n",
       "      <td>0.031006250621699986</td>\n",
       "      <td>-0.008798382997400003</td>\n",
       "      <td>0.009004523887710001</td>\n",
       "      <td>-0.05045604444349996</td>\n",
       "      <td>0.483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stddev</td>\n",
       "      <td>288.8194360957494</td>\n",
       "      <td>1.8889592815005751</td>\n",
       "      <td>2.394186945735675</td>\n",
       "      <td>2.0559674126799785</td>\n",
       "      <td>1.863208938242715</td>\n",
       "      <td>1.6208350027396687</td>\n",
       "      <td>1.5317579058984108</td>\n",
       "      <td>1.7728344372619638</td>\n",
       "      <td>1.9335911601925522</td>\n",
       "      <td>...</td>\n",
       "      <td>1.310014162316104</td>\n",
       "      <td>0.9928149233047021</td>\n",
       "      <td>1.3489092513130678</td>\n",
       "      <td>1.0175306226012728</td>\n",
       "      <td>0.9937576399485477</td>\n",
       "      <td>1.2158726897277283</td>\n",
       "      <td>1.4720581081411457</td>\n",
       "      <td>0.8531823148864874</td>\n",
       "      <td>1.0285753546436174</td>\n",
       "      <td>0.4999609594367951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>min</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.0021861139000000003</td>\n",
       "      <td>-0.00099003483</td>\n",
       "      <td>-0.007215624499999999</td>\n",
       "      <td>-0.0087255505</td>\n",
       "      <td>-0.0049381842</td>\n",
       "      <td>-0.0030899136</td>\n",
       "      <td>-0.0012198847</td>\n",
       "      <td>-0.0017179298999999999</td>\n",
       "      <td>-0.00151381</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max</td>\n",
       "      <td>999</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>4.247343099999999</td>\n",
       "      <td>3.266212</td>\n",
       "      <td>3.7208654</td>\n",
       "      <td>3.1033252</td>\n",
       "      <td>3.1775257999999997</td>\n",
       "      <td>3.5107522000000007</td>\n",
       "      <td>5.6920133</td>\n",
       "      <td>3.0476226000000004</td>\n",
       "      <td>3.1554324</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  summary               # id                 X_0                X_1  \\\n",
       "0   count               1000                1000               1000   \n",
       "1    mean              499.5               2.049              3.051   \n",
       "2  stddev  288.8194360957494  1.8889592815005751  2.394186945735675   \n",
       "3     min                  0                   0                  0   \n",
       "4     max                999                   6                  6   \n",
       "\n",
       "                  X_2                X_3                 X_4  \\\n",
       "0                1000               1000                1000   \n",
       "1               2.815              3.118               4.211   \n",
       "2  2.0559674126799785  1.863208938242715  1.6208350027396687   \n",
       "3                   0                  0                   1   \n",
       "4                   6                  6                   6   \n",
       "\n",
       "                  X_5                 X_6                 X_7  ...  \\\n",
       "0                1000                1000                1000  ...   \n",
       "1               3.992               3.351               2.658  ...   \n",
       "2  1.5317579058984108  1.7728344372619638  1.9335911601925522  ...   \n",
       "3                   2                   1                   0  ...   \n",
       "4                   6                   6                   6  ...   \n",
       "\n",
       "                     X_91                X_92                   X_93  \\\n",
       "0                    1000                1000                   1000   \n",
       "1    0.029485678112799993  -0.015578821739792  -0.034382649559500045   \n",
       "2       1.310014162316104  0.9928149233047021     1.3489092513130678   \n",
       "3  -0.0021861139000000003      -0.00099003483  -0.007215624499999999   \n",
       "4       4.247343099999999            3.266212              3.7208654   \n",
       "\n",
       "                  X_94                  X_95                  X_96  \\\n",
       "0                 1000                  1000                  1000   \n",
       "1  0.03410696849619999  0.004251966935300004  0.031006250621699986   \n",
       "2   1.0175306226012728    0.9937576399485477    1.2158726897277283   \n",
       "3        -0.0087255505         -0.0049381842         -0.0030899136   \n",
       "4            3.1033252    3.1775257999999997    3.5107522000000007   \n",
       "\n",
       "                    X_97                    X_98                  X_99  \\\n",
       "0                   1000                    1000                  1000   \n",
       "1  -0.008798382997400003    0.009004523887710001  -0.05045604444349996   \n",
       "2     1.4720581081411457      0.8531823148864874    1.0285753546436174   \n",
       "3          -0.0012198847  -0.0017179298999999999           -0.00151381   \n",
       "4              5.6920133      3.0476226000000004             3.1554324   \n",
       "\n",
       "                    y  \n",
       "0                1000  \n",
       "1               0.483  \n",
       "2  0.4999609594367951  \n",
       "3                   0  \n",
       "4                   1  \n",
       "\n",
       "[5 rows x 103 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计原数据集\n",
    "submitter.get_statistics(path=dataset_uri, file_type=FILE_TYPE_CSV,with_header=True).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:02:23,231 - INFO - request: http://localhost:8000/v1/evaluation/statistics/ with data {'job_id': 10024, 'type': 'csv', 'method': 'basic', 'with_header': True, 'from_sampling': True}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th># id</th>\n",
       "      <th>X_0</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_2</th>\n",
       "      <th>X_3</th>\n",
       "      <th>X_4</th>\n",
       "      <th>X_5</th>\n",
       "      <th>X_6</th>\n",
       "      <th>X_7</th>\n",
       "      <th>...</th>\n",
       "      <th>X_91</th>\n",
       "      <th>X_92</th>\n",
       "      <th>X_93</th>\n",
       "      <th>X_94</th>\n",
       "      <th>X_95</th>\n",
       "      <th>X_96</th>\n",
       "      <th>X_97</th>\n",
       "      <th>X_98</th>\n",
       "      <th>X_99</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>count</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>...</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean</td>\n",
       "      <td>514.8981481481482</td>\n",
       "      <td>2.25</td>\n",
       "      <td>3.0277777777777777</td>\n",
       "      <td>2.9166666666666665</td>\n",
       "      <td>2.8981481481481484</td>\n",
       "      <td>4.398148148148148</td>\n",
       "      <td>4.185185185185185</td>\n",
       "      <td>3.425925925925926</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.12504391845462962</td>\n",
       "      <td>-0.037750146018518504</td>\n",
       "      <td>-0.10642114362037043</td>\n",
       "      <td>0.08131659404166666</td>\n",
       "      <td>0.11069304162962962</td>\n",
       "      <td>0.09582038964351856</td>\n",
       "      <td>-0.1807713333055556</td>\n",
       "      <td>0.10353172625277773</td>\n",
       "      <td>-5.714228148147963E-4</td>\n",
       "      <td>0.49074074074074076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stddev</td>\n",
       "      <td>283.16568050058794</td>\n",
       "      <td>1.860182887200013</td>\n",
       "      <td>2.3539917711340834</td>\n",
       "      <td>2.005250118409965</td>\n",
       "      <td>1.839035783599934</td>\n",
       "      <td>1.5940298359923883</td>\n",
       "      <td>1.4670725389512356</td>\n",
       "      <td>1.8148077507085545</td>\n",
       "      <td>1.8444156686816837</td>\n",
       "      <td>...</td>\n",
       "      <td>1.2498019518135066</td>\n",
       "      <td>0.8767414243685469</td>\n",
       "      <td>1.3092069478023307</td>\n",
       "      <td>0.9532536404715464</td>\n",
       "      <td>0.9255972992810526</td>\n",
       "      <td>1.1801704496779792</td>\n",
       "      <td>1.3180501912308957</td>\n",
       "      <td>0.7724878924299564</td>\n",
       "      <td>1.077355924914047</td>\n",
       "      <td>0.5022448740055658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>min</td>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.022213316</td>\n",
       "      <td>-0.04997012</td>\n",
       "      <td>-0.0076247096</td>\n",
       "      <td>-0.0087255505</td>\n",
       "      <td>-0.027446876000000002</td>\n",
       "      <td>-0.0030899136</td>\n",
       "      <td>-0.024749287000000002</td>\n",
       "      <td>-0.0020981927</td>\n",
       "      <td>-0.0340015</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max</td>\n",
       "      <td>997</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5984216</td>\n",
       "      <td>2.1624603</td>\n",
       "      <td>3.7208654</td>\n",
       "      <td>3.1033252</td>\n",
       "      <td>2.438946</td>\n",
       "      <td>2.3365866000000004</td>\n",
       "      <td>2.6220742</td>\n",
       "      <td>1.6186575</td>\n",
       "      <td>2.0121164</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  summary                # id                X_0                 X_1  \\\n",
       "0   count                 108                108                 108   \n",
       "1    mean   514.8981481481482               2.25  3.0277777777777777   \n",
       "2  stddev  283.16568050058794  1.860182887200013  2.3539917711340834   \n",
       "3     min                 114                  0                   0   \n",
       "4     max                 997                  6                   6   \n",
       "\n",
       "                  X_2                 X_3                 X_4  \\\n",
       "0                 108                 108                 108   \n",
       "1  2.9166666666666665  2.8981481481481484   4.398148148148148   \n",
       "2   2.005250118409965   1.839035783599934  1.5940298359923883   \n",
       "3                   0                   0                   1   \n",
       "4                   6                   6                   6   \n",
       "\n",
       "                  X_5                 X_6                 X_7  ...  \\\n",
       "0                 108                 108                 108  ...   \n",
       "1   4.185185185185185   3.425925925925926                 3.0  ...   \n",
       "2  1.4670725389512356  1.8148077507085545  1.8444156686816837  ...   \n",
       "3                   2                   1                   0  ...   \n",
       "4                   6                   6                   6  ...   \n",
       "\n",
       "                  X_91                   X_92                  X_93  \\\n",
       "0                  108                    108                   108   \n",
       "1  0.12504391845462962  -0.037750146018518504  -0.10642114362037043   \n",
       "2   1.2498019518135066     0.8767414243685469    1.3092069478023307   \n",
       "3         -0.022213316            -0.04997012         -0.0076247096   \n",
       "4            2.5984216              2.1624603             3.7208654   \n",
       "\n",
       "                  X_94                   X_95                 X_96  \\\n",
       "0                  108                    108                  108   \n",
       "1  0.08131659404166666    0.11069304162962962  0.09582038964351856   \n",
       "2   0.9532536404715464     0.9255972992810526   1.1801704496779792   \n",
       "3        -0.0087255505  -0.027446876000000002        -0.0030899136   \n",
       "4            3.1033252               2.438946   2.3365866000000004   \n",
       "\n",
       "                    X_97                 X_98                   X_99  \\\n",
       "0                    108                  108                    108   \n",
       "1    -0.1807713333055556  0.10353172625277773  -5.714228148147963E-4   \n",
       "2     1.3180501912308957   0.7724878924299564      1.077355924914047   \n",
       "3  -0.024749287000000002        -0.0020981927             -0.0340015   \n",
       "4              2.6220742            1.6186575              2.0121164   \n",
       "\n",
       "                     y  \n",
       "0                  108  \n",
       "1  0.49074074074074076  \n",
       "2   0.5022448740055658  \n",
       "3                    0  \n",
       "4                    1  \n",
       "\n",
       "[5 rows x 103 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计抽样后的数据集\n",
    "data = submitter.get_statistics(job_id=job_id, from_sampling=True, file_type=FILE_TYPE_CSV,with_header=True).to_pandas()\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:02:29,415 - INFO - request: http://localhost:8000/v1/evaluation/job/ with data {'method': 'compare', 'type': 'csv', 'compare_job_id': 10024}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'code': 0, 'msg': '', 'data': {'job_id': 50004}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "50004"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提交评估任务\n",
    "cmp_evaluation_job = submitter.submit_evaluation_job(compare_job_id=job_id, file_type=FILE_TYPE_CSV)\n",
    "print(cmp_evaluation_job.to_dict())\n",
    "cmp_evaluation_job_id = cmp_evaluation_job.job_id\n",
    "cmp_evaluation_job_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:02:37,726 - INFO - request: http://localhost:8000/v1/query/evaluation/job/ with data {'job_id': 50004}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># id</th>\n",
       "      <th>X_0</th>\n",
       "      <th>X_1</th>\n",
       "      <th>X_10</th>\n",
       "      <th>X_11</th>\n",
       "      <th>X_12</th>\n",
       "      <th>X_13</th>\n",
       "      <th>X_14</th>\n",
       "      <th>X_15</th>\n",
       "      <th>X_16</th>\n",
       "      <th>...</th>\n",
       "      <th>X_91</th>\n",
       "      <th>X_92</th>\n",
       "      <th>X_93</th>\n",
       "      <th>X_94</th>\n",
       "      <th>X_95</th>\n",
       "      <th>X_96</th>\n",
       "      <th>X_97</th>\n",
       "      <th>X_98</th>\n",
       "      <th>X_99</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>...</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>514.8981481481482</td>\n",
       "      <td>2.25</td>\n",
       "      <td>3.0277777777777777</td>\n",
       "      <td>2.2777777777777777</td>\n",
       "      <td>3.0833333333333335</td>\n",
       "      <td>3.3796296296296298</td>\n",
       "      <td>2.0833333333333335</td>\n",
       "      <td>2.314814814814815</td>\n",
       "      <td>0.26768048228611113</td>\n",
       "      <td>-0.1807713333055556</td>\n",
       "      <td>...</td>\n",
       "      <td>0.12504391845462962</td>\n",
       "      <td>-0.037750146018518504</td>\n",
       "      <td>-0.10642114362037043</td>\n",
       "      <td>0.08131659404166666</td>\n",
       "      <td>0.11069304162962962</td>\n",
       "      <td>0.09582038964351856</td>\n",
       "      <td>-0.1807713333055556</td>\n",
       "      <td>0.10353172625277773</td>\n",
       "      <td>-5.714228148147963E-4</td>\n",
       "      <td>0.49074074074074076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev</th>\n",
       "      <td>283.16568050058794</td>\n",
       "      <td>1.860182887200013</td>\n",
       "      <td>2.3539917711340834</td>\n",
       "      <td>1.828299518056562</td>\n",
       "      <td>1.8144024015543951</td>\n",
       "      <td>2.0265416081544365</td>\n",
       "      <td>1.9389032452375037</td>\n",
       "      <td>1.4892019323424508</td>\n",
       "      <td>0.9582508085570517</td>\n",
       "      <td>1.3180501912308957</td>\n",
       "      <td>...</td>\n",
       "      <td>1.2498019518135066</td>\n",
       "      <td>0.8767414243685469</td>\n",
       "      <td>1.3092069478023307</td>\n",
       "      <td>0.9532536404715464</td>\n",
       "      <td>0.9255972992810526</td>\n",
       "      <td>1.1801704496779792</td>\n",
       "      <td>1.3180501912308957</td>\n",
       "      <td>0.7724878924299564</td>\n",
       "      <td>1.077355924914047</td>\n",
       "      <td>0.5022448740055658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.0026964582</td>\n",
       "      <td>-0.024749287000000002</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.022213316</td>\n",
       "      <td>-0.04997012</td>\n",
       "      <td>-0.0076247096</td>\n",
       "      <td>-0.0087255505</td>\n",
       "      <td>-0.027446876000000002</td>\n",
       "      <td>-0.0030899136</td>\n",
       "      <td>-0.024749287000000002</td>\n",
       "      <td>-0.0020981927</td>\n",
       "      <td>-0.0340015</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>997</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>2.3059315</td>\n",
       "      <td>2.6220742</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5984216</td>\n",
       "      <td>2.1624603</td>\n",
       "      <td>3.7208654</td>\n",
       "      <td>3.1033252</td>\n",
       "      <td>2.438946</td>\n",
       "      <td>2.3365866000000004</td>\n",
       "      <td>2.6220742</td>\n",
       "      <td>1.6186575</td>\n",
       "      <td>2.0121164</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_bias</th>\n",
       "      <td>0.0308271</td>\n",
       "      <td>0.0980966</td>\n",
       "      <td>0.00761135</td>\n",
       "      <td>0.0979098</td>\n",
       "      <td>0.0123852</td>\n",
       "      <td>0.0962146</td>\n",
       "      <td>0.0590184</td>\n",
       "      <td>0.0334802</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.988675</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev_bias</th>\n",
       "      <td>0.0195754</td>\n",
       "      <td>0.015234</td>\n",
       "      <td>0.0167887</td>\n",
       "      <td>0.00740913</td>\n",
       "      <td>0.032143</td>\n",
       "      <td>0.0236702</td>\n",
       "      <td>0.00936722</td>\n",
       "      <td>0.058047</td>\n",
       "      <td>0.00838547</td>\n",
       "      <td>0.104621</td>\n",
       "      <td>...</td>\n",
       "      <td>0.045963</td>\n",
       "      <td>0.116914</td>\n",
       "      <td>0.0294329</td>\n",
       "      <td>0.0631696</td>\n",
       "      <td>0.0685885</td>\n",
       "      <td>0.0293635</td>\n",
       "      <td>0.104621</td>\n",
       "      <td>0.0945805</td>\n",
       "      <td>0.0474254</td>\n",
       "      <td>0.00456819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>98.7399</td>\n",
       "      <td>97.1667</td>\n",
       "      <td>99.39</td>\n",
       "      <td>97.367</td>\n",
       "      <td>98.8868</td>\n",
       "      <td>97.0029</td>\n",
       "      <td>98.2904</td>\n",
       "      <td>97.7118</td>\n",
       "      <td>74.7904</td>\n",
       "      <td>72.3845</td>\n",
       "      <td>...</td>\n",
       "      <td>73.8509</td>\n",
       "      <td>72.0772</td>\n",
       "      <td>74.2642</td>\n",
       "      <td>73.4208</td>\n",
       "      <td>73.2853</td>\n",
       "      <td>74.2659</td>\n",
       "      <td>72.3845</td>\n",
       "      <td>72.6355</td>\n",
       "      <td>74.0975</td>\n",
       "      <td>99.8858</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           # id                X_0                 X_1  \\\n",
       "count                       108                108                 108   \n",
       "mean          514.8981481481482               2.25  3.0277777777777777   \n",
       "stddev       283.16568050058794  1.860182887200013  2.3539917711340834   \n",
       "min                         114                  0                   0   \n",
       "max                         997                  6                   6   \n",
       "mean_bias             0.0308271          0.0980966          0.00761135   \n",
       "stddev_bias           0.0195754           0.015234           0.0167887   \n",
       "score                   98.7399            97.1667               99.39   \n",
       "\n",
       "                           X_10                X_11                X_12  \\\n",
       "count                       108                 108                 108   \n",
       "mean         2.2777777777777777  3.0833333333333335  3.3796296296296298   \n",
       "stddev        1.828299518056562  1.8144024015543951  2.0265416081544365   \n",
       "min                           0                   1                   0   \n",
       "max                           5                   6                   6   \n",
       "mean_bias             0.0979098           0.0123852           0.0962146   \n",
       "stddev_bias          0.00740913            0.032143           0.0236702   \n",
       "score                    97.367             98.8868             97.0029   \n",
       "\n",
       "                           X_13                X_14                 X_15  \\\n",
       "count                       108                 108                  108   \n",
       "mean         2.0833333333333335   2.314814814814815  0.26768048228611113   \n",
       "stddev       1.9389032452375037  1.4892019323424508   0.9582508085570517   \n",
       "min                           0                   0        -0.0026964582   \n",
       "max                           6                   5            2.3059315   \n",
       "mean_bias             0.0590184           0.0334802                    1   \n",
       "stddev_bias          0.00936722            0.058047           0.00838547   \n",
       "score                   98.2904             97.7118              74.7904   \n",
       "\n",
       "                              X_16  ...                 X_91  \\\n",
       "count                          108  ...                  108   \n",
       "mean           -0.1807713333055556  ...  0.12504391845462962   \n",
       "stddev          1.3180501912308957  ...   1.2498019518135066   \n",
       "min          -0.024749287000000002  ...         -0.022213316   \n",
       "max                      2.6220742  ...            2.5984216   \n",
       "mean_bias                        1  ...                    1   \n",
       "stddev_bias               0.104621  ...             0.045963   \n",
       "score                      72.3845  ...              73.8509   \n",
       "\n",
       "                              X_92                  X_93                 X_94  \\\n",
       "count                          108                   108                  108   \n",
       "mean         -0.037750146018518504  -0.10642114362037043  0.08131659404166666   \n",
       "stddev          0.8767414243685469    1.3092069478023307   0.9532536404715464   \n",
       "min                    -0.04997012         -0.0076247096        -0.0087255505   \n",
       "max                      2.1624603             3.7208654            3.1033252   \n",
       "mean_bias                        1                     1                    1   \n",
       "stddev_bias               0.116914             0.0294329            0.0631696   \n",
       "score                      72.0772               74.2642              73.4208   \n",
       "\n",
       "                              X_95                 X_96  \\\n",
       "count                          108                  108   \n",
       "mean           0.11069304162962962  0.09582038964351856   \n",
       "stddev          0.9255972992810526   1.1801704496779792   \n",
       "min          -0.027446876000000002        -0.0030899136   \n",
       "max                       2.438946   2.3365866000000004   \n",
       "mean_bias                        1                    1   \n",
       "stddev_bias              0.0685885            0.0293635   \n",
       "score                      73.2853              74.2659   \n",
       "\n",
       "                              X_97                 X_98  \\\n",
       "count                          108                  108   \n",
       "mean           -0.1807713333055556  0.10353172625277773   \n",
       "stddev          1.3180501912308957   0.7724878924299564   \n",
       "min          -0.024749287000000002        -0.0020981927   \n",
       "max                      2.6220742            1.6186575   \n",
       "mean_bias                        1                    1   \n",
       "stddev_bias               0.104621            0.0945805   \n",
       "score                      72.3845              72.6355   \n",
       "\n",
       "                              X_99                    y  \n",
       "count                          108                  108  \n",
       "mean         -5.714228148147963E-4  0.49074074074074076  \n",
       "stddev           1.077355924914047   0.5022448740055658  \n",
       "min                     -0.0340015                    0  \n",
       "max                      2.0121164                    1  \n",
       "mean_bias                 0.988675                    0  \n",
       "stddev_bias              0.0474254           0.00456819  \n",
       "score                      74.0975              99.8858  \n",
       "\n",
       "[8 rows x 102 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看各属性评估得分\n",
    "cmp_evaluation_job_data = submitter.get_evaluation_job_details(job_id=cmp_evaluation_job_id)\n",
    "cmp_df = cmp_evaluation_job_data.to_pandas()\n",
    "cmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73.15674482498201"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "cmp_df = cmp_df[selected_features_list]\n",
    "score_list = cmp_df.loc['score'].to_list()\n",
    "while -1 in score_list:\n",
    "    score_list.remove(-1)\n",
    "score_list\n",
    "import numpy as np\n",
    "np.mean(score_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:02:45,644 - INFO - request: http://localhost:8000/v1/evaluation/job/ with data {'method': 'kmeans', 'type': 'csv', 'compare_job_id': 10024, 'key': 'y', 'selected_features_list': ['X_20', 'X_80']}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'code': 0, 'msg': '', 'data': {'job_id': 50005}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "50005"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km_conf = {\n",
    "    \"compare_job_id\": job_id,\n",
    "    \"type\": \"csv\",\n",
    "    \"method\": \"kmeans\",\n",
    "    \"key\": \"y\",\n",
    "    \"selected_features_list\": selected_features_list\n",
    "}\n",
    "# 提交评估任务\n",
    "km_evaluation_job = submitter.submit_evaluation_job(**km_conf)\n",
    "print(km_evaluation_job.to_dict())\n",
    "km_evaluation_job_id = km_evaluation_job.job_id\n",
    "km_evaluation_job_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-29 18:03:08,961 - INFO - request: http://localhost:8000/v1/query/evaluation/job/ with data {'job_id': 50005}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'score': 99.0, 'accuracy': 0.9741379310344828, 'centers_result': 100}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看各属性评估得分\n",
    "km_evaluation_job_data = submitter.get_evaluation_job_details(job_id=km_evaluation_job_id)\n",
    "km_score = km_evaluation_job_data.result\n",
    "km_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       517\n",
      "           1       1.00      1.00      1.00       483\n",
      "\n",
      "    accuracy                           1.00      1000\n",
      "   macro avg       1.00      1.00      1.00      1000\n",
      "weighted avg       1.00      1.00      1.00      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 简单的读取抽样之后的文件\n",
    "from pyspark.sql import SparkSession\n",
    "from sparksampling.config import SPARK_CONF\n",
    "\n",
    "conf = SPARK_CONF\n",
    "spark = SparkSession.builder.config(conf=conf).getOrCreate()\n",
    "df = spark.read.csv(dataset_uri, header=True).toPandas()\n",
    "# 可以在这后面做数据分析，或试试看下面的统计、评估功能\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "train_y = df[label_index]\n",
    "train_X = df[selected_features_list]\n",
    "# train_X = df.drop([\"# id\"], axis=1)\n",
    "model.fit(train_X,train_y)\n",
    "tsdf = spark.read.csv(dataset_uri, header=True)\n",
    "tdf = tsdf.toPandas()\n",
    "\n",
    "test_y = tdf[label_index]\n",
    "test_X = tdf[train_X.columns]\n",
    "# test_X = test_X[feature_list]\n",
    "pred_y = model.predict(test_X)\n",
    "# data analyse here\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_true=test_y, y_pred=pred_y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}